from scrapy import signals
from scrapy.http import HtmlResponse
from selenium import webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import logging


class SeleniumMiddleware:
       #selenium中间件类，用于处理需要JavaScript渲染的动态网页


    def __init__(self):#初始化方法，创建浏览器驱动实例
        #使用绝对路径给出驱动位置
        edge_driver_path = r'C:\Users\lenovo\anaconda3\envs\py311\Scripts\msedgedriver.exe'
        service = Service(edge_driver_path)
        self.driver = webdriver.Edge(service=service)
        self.logger = logging.getLogger(__name__)
        self.logger.info(f"已使用路径 {edge_driver_path} 初始化驱动")

    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        crawler.signals.connect(middleware.spider_closed, signal=signals.spider_closed)
        return middleware

    def process_request(self, request, spider):
        # 检查请求URL是否包含'eastmoney.com'，只处理东方财富网的请求
        if 'eastmoney.com' in request.url:
            self.driver.get(request.url)

            try:
                wait = WebDriverWait(self.driver, 15)
                wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "table")))
                self.logger.info("股票数据表格已加载")
            except Exception as e:
                spider.logger.warning(f"等待数据加载超时: {e}")

            time.sleep(3)

            body = self.driver.page_source.encode('utf-8')
            return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)

    def spider_closed(self, spider):
        self.logger.info("正在关闭浏览器...")
        if hasattr(self, 'driver'):
            self.driver.quit()
            self.logger.info("浏览器已关闭")